!pip install -q torch datasets transformers[torch] spacy[cuda11x] spacy_fastlang lime shap

import pickle

import numpy as np
import pandas as pd
import spacy
import spacy_fastlang
import torch
import shap
from datasets import Dataset, DatasetDict
from scipy.stats import binomtest
from scipy.stats.distributions import loguniform
from sklearn.dummy import DummyClassifier
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import (
    accuracy_score,
    f1_score,
    make_scorer,
    top_k_accuracy_score,
)
from sklearn.metrics import  classification_report
from sklearn.model_selection import (
    RandomizedSearchCV,
    StratifiedShuffleSplit,
    train_test_split,
)
from sklearn.pipeline import make_pipeline
from sklearn.svm import LinearSVC
from tqdm.auto import tqdm
from transformers.pipelines.pt_utils import KeyDataset

#spacy.require_gpu()

tqdm.pandas()
pd.options.mode.copy_on_write = True
pd.options.display.max_colwidth = 1000

with open("classifier.pkl", "rb") as f:
    classifier = pickle.load(f)
    
torch.set_float32_matmul_precision("high")
print(classifier.model)

RobertaForSequenceClassification(
  (roberta): RobertaModel(
    (embeddings): RobertaEmbeddings(
      (word_embeddings): Embedding(50265, 768, padding_idx=1)
      (position_embeddings): Embedding(514, 768, padding_idx=1)
      (token_type_embeddings): Embedding(1, 768)
      (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
      (dropout): Dropout(p=0.1, inplace=False)
    )
    (encoder): RobertaEncoder(
      (layer): ModuleList(
        (0-11): 12 x RobertaLayer(
          (attention): RobertaAttention(
            (self): RobertaSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
            (output): RobertaSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
          )
          (intermediate): RobertaIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
            (intermediate_act_fn): GELUActivation()
          )
          (output): RobertaOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
        )
      )
    )
  )
  (classifier): RobertaClassificationHead(
    (dense): Linear(in_features=768, out_features=768, bias=True)
    (dropout): Dropout(p=0.1, inplace=False)
    (out_proj): Linear(in_features=768, out_features=4, bias=True)
  )
)

from datasets import load_from_disk
dataset = load_from_disk("Project2_data")

train_df = pd.read_pickle('Project2_train_data.pkl.gz')
test_df = pd.read_pickle('Project2_test_data.pkl.gz')
valid_df = pd.read_pickle('Project2_valid_data.pkl.gz')

train_df ["label"] = train_df ["wine_group"]
train_df ["text"] = train_df ["review_text"]
test_df ["label"] = test_df ["wine_group"]
test_df ["text"] = test_df ["review_text"]
valid_df ["label"] = valid_df ["wine_group"]
valid_df ["text"] = valid_df ["review_text"]

dataset = DatasetDict()
dataset["train"] = Dataset.from_pandas(train_df)
dataset["valid"] = Dataset.from_pandas(valid_df)
dataset["test"] = Dataset.from_pandas(test_df)


dataset = dataset.class_encode_column("label")

#dataset.save_to_disk("lab09_dataset")

Casting to class labels:   0%|          | 0/7500 [00:00<?, ? examples/s]

Casting to class labels:   0%|          | 0/5000 [00:00<?, ? examples/s]

Casting to class labels:   0%|          | 0/2500 [00:00<?, ? examples/s]

def evaluate(pipeline, dataset, batch_size=32):
    results = list(
        tqdm(
            pipeline(KeyDataset(dataset, "text"), batch_size=32),
            total=len(dataset),
        )
    )

    predicted = [r["label"] for r in results]
    labels = sorted(set(predicted))
    scores = np.vstack(
        [
            [r[label] for label in labels]
            for r in [{r["label"]: r["score"] for r in items} for items in results]
        ]
    )

    return predicted, scores

labels = sorted(set(predicted))
labels

['Bold Reds',
 'Full-bodied Whites',
 'Light-bodied, Crisp Whites',
 'Medium to Full-bodied Reds']

#valid_df = pd.read_pickle('Project2_valid_data.pkl.gz')
valid_df['predicted'] = predicted
valid_df.head()

errors = valid_df.query('wine_group != predicted').sample(30)

import shap

explainer = shap.Explainer(shap.models.TransformersPipeline(classifier,rescale_to_logits=True), output_names=labels)
shap_values = explainer(KeyDataset(Dataset.from_pandas(errors), "review_text"))

  0%|          | 0/498 [00:00<?, ?it/s]

PartitionExplainer explainer:   3%|▎         | 1/30 [00:00<?, ?it/s]

  0%|          | 0/498 [00:00<?, ?it/s]

PartitionExplainer explainer:  10%|█         | 3/30 [00:18<01:51,  4.13s/it]

  0%|          | 0/498 [00:00<?, ?it/s]

PartitionExplainer explainer:  13%|█▎        | 4/30 [00:26<02:32,  5.86s/it]

  0%|          | 0/498 [00:00<?, ?it/s]

PartitionExplainer explainer:  17%|█▋        | 5/30 [00:35<02:49,  6.77s/it]

  0%|          | 0/498 [00:00<?, ?it/s]

PartitionExplainer explainer:  20%|██        | 6/30 [00:43<02:54,  7.29s/it]

a = pd.DataFrame(errors["wine_group"].value_counts()).reset_index()
b = pd.DataFrame(errors["predicted"].value_counts()).reset_index()
c = pd.merge(a,b,left_on = "wine_group", right_on = "predicted", how = "left")
c["error_count"] = c['count_x'] + c['count_y']
c = c[["wine_group", "error_count"]]
c

errors["combination"] = errors['wine_group']+ "+" + errors['predicted']
comb_orig = pd.DataFrame(errors["combination"].value_counts()).reset_index()
comb_orig

errors = errors.rename_axis('key_orig').reset_index()
errors = errors.rename_axis('key').reset_index()
errors.head()

comb1 = errors[errors["combination"].isin(["Medium to Full-bodied Reds+Bold Reds"])][["key",'wine_group','predicted','review_text']]
print(f'There are {comb1.shape[0]} mislabeled combinations of "Medium to Full-bodied Reds+Bold Reds"')
comb1.head(2)

There are 11 mislabeled combinations of "Medium to Full-bodied Reds+Bold Reds"

for key in comb1["key"]:
    #print(f"for observation number - {comb1.iloc[:key]} :")
    shap.plots.text(shap_values[key])

comb2 = errors[errors["combination"].isin(["Bold Reds+Medium to Full-bodied Reds"])][["key",'wine_group','predicted','review_text']]
print(f'There are {comb2.shape[0]} mislabeled combinations of "Bold Reds+Medium to Full-bodied Reds"')
comb2.head(2)

There are 9 mislabeled combinations of "Bold Reds+Medium to Full-bodied Reds"

for key in comb2.index:
    shap.plots.text(shap_values[key])

comb2 = errors[errors["combination"].isin(["Light-bodied, Crisp Whites+Full-bodied Whites"])][["key",'wine_group','predicted','review_text']]
print(f'There are {comb2.shape[0]} mislabeled combinations of "Light-bodied, Crisp Whites+Full-bodied Whites"')
comb2.head(2)

There are 4 mislabeled combinations of "Light-bodied, Crisp Whites+Full-bodied Whites"

for key in comb2.index:
    shap.plots.text(shap_values[key])

comb2 = errors[errors["combination"].isin(["Full-bodied Whites+Light-bodied, Crisp Whites"])][["key",'wine_group','predicted','review_text']]
print(f'There are {comb2.shape[0]} mislabeled combinations of "Light-bodied, Crisp Whites+Full-bodied Whites"')
comb2

There are 3 mislabeled combinations of "Light-bodied, Crisp Whites+Full-bodied Whites"

for key in comb2.index:
    shap.plots.text(shap_values[key])

	wine_group	error_count
0	Medium to Full-bodied Reds	22
1	Bold Reds	21
2	Light-bodied, Crisp Whites	9
3	Full-bodied Whites	8

	combination	count
0	Medium to Full-bodied Reds+Bold Reds	11
1	Bold Reds+Medium to Full-bodied Reds	9
2	Light-bodied, Crisp Whites+Full-bodied Whites	4
3	Full-bodied Whites+Light-bodied, Crisp Whites	3
4	Medium to Full-bodied Reds+Light-bodied, Crisp Whites	1
5	Bold Reds+Light-bodied, Crisp Whites	1
6	Medium to Full-bodied Reds+Full-bodied Whites	1

	key	key_orig	review_text	wine_variant	wine_group	predicted	combination
0	0	27473	Popped and poured. Big fruit on the nose and taste. Medium finish. Very easy to drink and completely enjoyable!	Cabernet Sauvignon/Zinfandel	Bold Reds	Medium to Full-bodied Reds	Bold Reds+Medium to Full-bodied Reds
1	1	18471	Great fruit an acidity, this one could go a bit longer but the fruit is so nice right now, it is almost impossible to resist now.	Cabernet Sauvignon/Zinfandel	Bold Reds	Medium to Full-bodied Reds	Bold Reds+Medium to Full-bodied Reds
2	2	9500	Some barnyard funk when opened (not corked). This gradually blew off, but the fruit wasn't showing. In the mouth, soft and smooth, but no real structure left. Short finish.	Cabernet Sauvignon/Zinfandel	Bold Reds	Medium to Full-bodied Reds	Bold Reds+Medium to Full-bodied Reds
3	3	12308	Great wine with a very bad cork - it is a great bottle for the price ($25) but it is incredibkle how bad corcks came in this vintage. The good is that the wine is not affected by it.	Cabernet Sauvignon/Zinfandel	Bold Reds	Medium to Full-bodied Reds	Bold Reds+Medium to Full-bodied Reds
4	4	88	Very smooth with a lite acid finish. Gets better as it breathes. Complimented the Italian sausage and tomato basil sauce. Over fresh tri-color pasta	Cabernet Sauvignon/Zinfandel	Bold Reds	Medium to Full-bodied Reds	Bold Reds+Medium to Full-bodied Reds

	key_orig	key	wine_group	predicted	review_text
5	22357	5	Medium to Full-bodied Reds	Bold Reds	Powerful. Oak, blueberries and char on the nose. All in refrain on the palate with a light lingering oaky vapor trail.
6	9301	6	Medium to Full-bodied Reds	Bold Reds	Hmmm. I didn't have the experience the majority to the tasters here had. This was opened in the morning and I tried it that night. Maybe that had something to do with it. The nose was pleasant, with dark fruits and some richness felt. But the palate is very ripe, almost to the point of being syrupy, and it's hot and bitter on the finish. Way out of balance, without enough richness or fruit on the front palate to fill out the high alcohol. I was very impressed with their blends, too, so this wine surprised me.

	key	wine_group	predicted	review_text
19	19	Light-bodied, Crisp Whites	Full-bodied Whites	olden color, honey, apricot, a touch of orange. Sweet, syrapy. Lucious. Could use a touch more acidity.
25	25	Light-bodied, Crisp Whites	Full-bodied Whites	Almost tasted like champagne - which is always a plus. Sweet but not overly sweet, went very well with the shrimp and toasted couscous. Be sure to chill well before serving.

	key	wine_group	predicted	review_text
7	7	Full-bodied Whites	Light-bodied, Crisp Whites	Rich sweet honeysuckle, creamy mint, basil, brown sugar and peach all coming through on the nose. Velvety mouthfeel, nice acidity, good extraction.
11	11	Full-bodied Whites	Light-bodied, Crisp Whites	Luscious, honeysuckle, peach, and apricot on the nose. Quite a full mouth feel but balanced with wonderful acidity and minerality. Lovely wine, rich but not over-the-top. Yum.
17	17	Full-bodied Whites	Light-bodied, Crisp Whites	Nice moderate bright golden color. Nose shows green lemon grass, some herbaceous olive oil action coupled with a light baking spice aroma. The palate shows lots of acid and is extremely young and tightly wound. More citric yellow fruit - lemons, some tart lime and a bit of yellow apple. The finish is highly acidic and needs some time to come into itself. Yet this was a perfect wine to awaken the taste buds before dinner. Nice potential.

ERROR ANALYSIS¶